{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['oof_0.9235726706987148.csv', 'oof_cat_0.9180511510872795.csv',\n",
       "       'oof_lgb_0.9160928879646695.csv', 'oof_lgb_0.9224806732079125.csv',\n",
       "       'oof_lgb_0.9225898803415156.csv', 'oof_lgb_0.9229.csv',\n",
       "       'oof_lgb_0.92342.csv', 'oof_lgb_0.9237994961308414.csv',\n",
       "       'oof_lgb_0.923864316198583.csv', 'oof_xgb_0.9175107247524672.csv',\n",
       "       'sub_0.9235726706987148.csv', 'sub_cat_0.9180511510872795.csv',\n",
       "       'sub_lgb_0.9160928879646695.csv', 'sub_lgb_0.9224806732079125.csv',\n",
       "       'sub_lgb_0.9225898803415156.csv', 'sub_lgb_0.9229.csv',\n",
       "       'sub_lgb_0.92342.csv', 'sub_lgb_0.9237994961308414.csv',\n",
       "       'sub_lgb_0.923864316198583.csv', 'sub_xgb_0.9175107247524672.csv'],\n",
       "      dtype='<U30')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression, LinearRegression\n",
    "\n",
    "np.sort(os.listdir('./subs'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = list(np.sort([c for c in (os.listdir('./subs')) if 'sub' in c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sub_0.9235726706987148.csv',\n",
       " 'sub_cat_0.9180511510872795.csv',\n",
       " 'sub_lgb_0.9160928879646695.csv',\n",
       " 'sub_lgb_0.9224806732079125.csv',\n",
       " 'sub_lgb_0.9225898803415156.csv',\n",
       " 'sub_lgb_0.9229.csv',\n",
       " 'sub_lgb_0.92342.csv',\n",
       " 'sub_lgb_0.9237994961308414.csv',\n",
       " 'sub_lgb_0.923864316198583.csv',\n",
       " 'sub_xgb_0.9175107247524672.csv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "oof = list(np.sort([c for c in (os.listdir('./subs')) if 'oof' in c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = pd.read_csv('./input/train.csv')['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oof_0.9235726706987148.csv \t auc :  0.9235726706987148\n",
      "oof_cat_0.9180511510872795.csv \t auc :  0.9180511510872796\n",
      "oof_lgb_0.9160928879646695.csv \t auc :  0.9160928879646695\n",
      "oof_lgb_0.9224806732079125.csv \t auc :  0.9224806732079125\n",
      "oof_lgb_0.9225898803415156.csv \t auc :  0.9225898803415156\n",
      "oof_lgb_0.9229.csv \t auc :  0.9229102322743912\n",
      "oof_lgb_0.92342.csv \t auc :  0.9234290920692649\n",
      "oof_lgb_0.9237994961308414.csv \t auc :  0.9237994961308414\n",
      "oof_lgb_0.923864316198583.csv \t auc :  0.923864316198583\n",
      "oof_xgb_0.9175107247524672.csv \t auc :  0.9175107247524672\n"
     ]
    }
   ],
   "source": [
    "val = pd.DataFrame()\n",
    "for j,i in enumerate(oof):\n",
    "    val['m{}'.format(j)] = pd.read_csv(os.path.join('./subs',i))['oof']\n",
    "    print(i,'\\t auc : ', roc_auc_score(target,val['m{}'.format(j)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>m0</th>\n",
       "      <th>m1</th>\n",
       "      <th>m2</th>\n",
       "      <th>m3</th>\n",
       "      <th>m4</th>\n",
       "      <th>m5</th>\n",
       "      <th>m6</th>\n",
       "      <th>m7</th>\n",
       "      <th>m8</th>\n",
       "      <th>m9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>m0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.976275</td>\n",
       "      <td>0.973499</td>\n",
       "      <td>0.987629</td>\n",
       "      <td>0.989253</td>\n",
       "      <td>0.988651</td>\n",
       "      <td>0.979474</td>\n",
       "      <td>0.980428</td>\n",
       "      <td>0.979630</td>\n",
       "      <td>0.976052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m1</th>\n",
       "      <td>0.976275</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.952891</td>\n",
       "      <td>0.973310</td>\n",
       "      <td>0.974209</td>\n",
       "      <td>0.973836</td>\n",
       "      <td>0.956688</td>\n",
       "      <td>0.957289</td>\n",
       "      <td>0.956413</td>\n",
       "      <td>0.964009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m2</th>\n",
       "      <td>0.973499</td>\n",
       "      <td>0.952891</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.966279</td>\n",
       "      <td>0.966821</td>\n",
       "      <td>0.961334</td>\n",
       "      <td>0.949708</td>\n",
       "      <td>0.950803</td>\n",
       "      <td>0.949619</td>\n",
       "      <td>0.956455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m3</th>\n",
       "      <td>0.987629</td>\n",
       "      <td>0.973310</td>\n",
       "      <td>0.966279</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996484</td>\n",
       "      <td>0.990229</td>\n",
       "      <td>0.975875</td>\n",
       "      <td>0.976782</td>\n",
       "      <td>0.975837</td>\n",
       "      <td>0.971604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m4</th>\n",
       "      <td>0.989253</td>\n",
       "      <td>0.974209</td>\n",
       "      <td>0.966821</td>\n",
       "      <td>0.996484</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.991869</td>\n",
       "      <td>0.979091</td>\n",
       "      <td>0.980040</td>\n",
       "      <td>0.979056</td>\n",
       "      <td>0.972762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m5</th>\n",
       "      <td>0.988651</td>\n",
       "      <td>0.973836</td>\n",
       "      <td>0.961334</td>\n",
       "      <td>0.990229</td>\n",
       "      <td>0.991869</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.983864</td>\n",
       "      <td>0.985204</td>\n",
       "      <td>0.983990</td>\n",
       "      <td>0.972332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m6</th>\n",
       "      <td>0.979474</td>\n",
       "      <td>0.956688</td>\n",
       "      <td>0.949708</td>\n",
       "      <td>0.975875</td>\n",
       "      <td>0.979091</td>\n",
       "      <td>0.983864</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995840</td>\n",
       "      <td>0.996303</td>\n",
       "      <td>0.958573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m7</th>\n",
       "      <td>0.980428</td>\n",
       "      <td>0.957289</td>\n",
       "      <td>0.950803</td>\n",
       "      <td>0.976782</td>\n",
       "      <td>0.980040</td>\n",
       "      <td>0.985204</td>\n",
       "      <td>0.995840</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996710</td>\n",
       "      <td>0.959340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m8</th>\n",
       "      <td>0.979630</td>\n",
       "      <td>0.956413</td>\n",
       "      <td>0.949619</td>\n",
       "      <td>0.975837</td>\n",
       "      <td>0.979056</td>\n",
       "      <td>0.983990</td>\n",
       "      <td>0.996303</td>\n",
       "      <td>0.996710</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m9</th>\n",
       "      <td>0.976052</td>\n",
       "      <td>0.964009</td>\n",
       "      <td>0.956455</td>\n",
       "      <td>0.971604</td>\n",
       "      <td>0.972762</td>\n",
       "      <td>0.972332</td>\n",
       "      <td>0.958573</td>\n",
       "      <td>0.959340</td>\n",
       "      <td>0.958197</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          m0        m1        m2        m3        m4        m5        m6  \\\n",
       "m0  1.000000  0.976275  0.973499  0.987629  0.989253  0.988651  0.979474   \n",
       "m1  0.976275  1.000000  0.952891  0.973310  0.974209  0.973836  0.956688   \n",
       "m2  0.973499  0.952891  1.000000  0.966279  0.966821  0.961334  0.949708   \n",
       "m3  0.987629  0.973310  0.966279  1.000000  0.996484  0.990229  0.975875   \n",
       "m4  0.989253  0.974209  0.966821  0.996484  1.000000  0.991869  0.979091   \n",
       "m5  0.988651  0.973836  0.961334  0.990229  0.991869  1.000000  0.983864   \n",
       "m6  0.979474  0.956688  0.949708  0.975875  0.979091  0.983864  1.000000   \n",
       "m7  0.980428  0.957289  0.950803  0.976782  0.980040  0.985204  0.995840   \n",
       "m8  0.979630  0.956413  0.949619  0.975837  0.979056  0.983990  0.996303   \n",
       "m9  0.976052  0.964009  0.956455  0.971604  0.972762  0.972332  0.958573   \n",
       "\n",
       "          m7        m8        m9  \n",
       "m0  0.980428  0.979630  0.976052  \n",
       "m1  0.957289  0.956413  0.964009  \n",
       "m2  0.950803  0.949619  0.956455  \n",
       "m3  0.976782  0.975837  0.971604  \n",
       "m4  0.980040  0.979056  0.972762  \n",
       "m5  0.985204  0.983990  0.972332  \n",
       "m6  0.995840  0.996303  0.958573  \n",
       "m7  1.000000  0.996710  0.959340  \n",
       "m8  0.996710  1.000000  0.958197  \n",
       "m9  0.959340  0.958197  1.000000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9245316314800366 0.002068595305655094\n"
     ]
    }
   ],
   "source": [
    "cv = cross_val_score(LinearRegression(), val, target, cv = 5, scoring = 'roc_auc')\n",
    "print(cv.mean(), cv.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 roc_auc_score :  0.9234984232280692\n",
      "Fold 1 roc_auc_score :  0.9234862069962022\n",
      "Fold 2 roc_auc_score :  0.9293184508253618\n",
      "Fold 3 roc_auc_score :  0.9241383583619829\n",
      "Fold 4 roc_auc_score :  0.9219864412799625\n",
      "roc_auc_score :  0.9244718016050045\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "models = []\n",
    "folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "oof = np.zeros(len(val))\n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(val.values, target.values)):\n",
    "\n",
    "    X_tr = val.iloc[trn_idx]\n",
    "    y_tr = target.iloc[trn_idx]\n",
    "    \n",
    "    X_val = val.iloc[val_idx]\n",
    "    y_val = target.iloc[val_idx]\n",
    "    model = LinearRegression()\n",
    "    model.fit(X_tr, y_tr)\n",
    "    models.append(model)\n",
    "    oof[val_idx] = model.predict(X_val)\n",
    "    print(\"Fold {}\".format(fold_),'roc_auc_score : ', roc_auc_score(y_val,oof[val_idx]))\n",
    "    \n",
    "score = roc_auc_score(target,oof)\n",
    "print('roc_auc_score : ', score)\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('./subs/'+pred[0])\n",
    "test.rename(columns={'target':'m0'},inplace = True)\n",
    "for j,i in enumerate(pred[1:]):\n",
    "    df = pd.read_csv(os.path.join('./subs',i))\n",
    "    df.rename(columns={'target':'m{}'.format(j+1)},inplace = True)\n",
    "    test = test.merge(df, on='ID_code')\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>m0</th>\n",
       "      <th>m1</th>\n",
       "      <th>m2</th>\n",
       "      <th>m3</th>\n",
       "      <th>m4</th>\n",
       "      <th>m5</th>\n",
       "      <th>m6</th>\n",
       "      <th>m7</th>\n",
       "      <th>m8</th>\n",
       "      <th>m9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>m0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.990650</td>\n",
       "      <td>0.768680</td>\n",
       "      <td>0.993359</td>\n",
       "      <td>0.994468</td>\n",
       "      <td>0.994312</td>\n",
       "      <td>0.985356</td>\n",
       "      <td>0.984884</td>\n",
       "      <td>0.984758</td>\n",
       "      <td>0.993996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m1</th>\n",
       "      <td>0.990650</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.760585</td>\n",
       "      <td>0.989072</td>\n",
       "      <td>0.989472</td>\n",
       "      <td>0.989305</td>\n",
       "      <td>0.973131</td>\n",
       "      <td>0.972206</td>\n",
       "      <td>0.972111</td>\n",
       "      <td>0.991001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m2</th>\n",
       "      <td>0.768680</td>\n",
       "      <td>0.760585</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.759885</td>\n",
       "      <td>0.761297</td>\n",
       "      <td>0.760598</td>\n",
       "      <td>0.760832</td>\n",
       "      <td>0.760664</td>\n",
       "      <td>0.760563</td>\n",
       "      <td>0.766020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m3</th>\n",
       "      <td>0.993359</td>\n",
       "      <td>0.989072</td>\n",
       "      <td>0.759885</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999429</td>\n",
       "      <td>0.995873</td>\n",
       "      <td>0.981843</td>\n",
       "      <td>0.981131</td>\n",
       "      <td>0.980880</td>\n",
       "      <td>0.991019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m4</th>\n",
       "      <td>0.994468</td>\n",
       "      <td>0.989472</td>\n",
       "      <td>0.761297</td>\n",
       "      <td>0.999429</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996905</td>\n",
       "      <td>0.984529</td>\n",
       "      <td>0.983897</td>\n",
       "      <td>0.983677</td>\n",
       "      <td>0.991447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m5</th>\n",
       "      <td>0.994312</td>\n",
       "      <td>0.989305</td>\n",
       "      <td>0.760598</td>\n",
       "      <td>0.995873</td>\n",
       "      <td>0.996905</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.989534</td>\n",
       "      <td>0.989031</td>\n",
       "      <td>0.988838</td>\n",
       "      <td>0.991499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m6</th>\n",
       "      <td>0.985356</td>\n",
       "      <td>0.973131</td>\n",
       "      <td>0.760832</td>\n",
       "      <td>0.981843</td>\n",
       "      <td>0.984529</td>\n",
       "      <td>0.989534</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999621</td>\n",
       "      <td>0.999607</td>\n",
       "      <td>0.978728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m7</th>\n",
       "      <td>0.984884</td>\n",
       "      <td>0.972206</td>\n",
       "      <td>0.760664</td>\n",
       "      <td>0.981131</td>\n",
       "      <td>0.983897</td>\n",
       "      <td>0.989031</td>\n",
       "      <td>0.999621</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999765</td>\n",
       "      <td>0.977965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m8</th>\n",
       "      <td>0.984758</td>\n",
       "      <td>0.972111</td>\n",
       "      <td>0.760563</td>\n",
       "      <td>0.980880</td>\n",
       "      <td>0.983677</td>\n",
       "      <td>0.988838</td>\n",
       "      <td>0.999607</td>\n",
       "      <td>0.999765</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.977677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m9</th>\n",
       "      <td>0.993996</td>\n",
       "      <td>0.991001</td>\n",
       "      <td>0.766020</td>\n",
       "      <td>0.991019</td>\n",
       "      <td>0.991447</td>\n",
       "      <td>0.991499</td>\n",
       "      <td>0.978728</td>\n",
       "      <td>0.977965</td>\n",
       "      <td>0.977677</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          m0        m1        m2        m3        m4        m5        m6  \\\n",
       "m0  1.000000  0.990650  0.768680  0.993359  0.994468  0.994312  0.985356   \n",
       "m1  0.990650  1.000000  0.760585  0.989072  0.989472  0.989305  0.973131   \n",
       "m2  0.768680  0.760585  1.000000  0.759885  0.761297  0.760598  0.760832   \n",
       "m3  0.993359  0.989072  0.759885  1.000000  0.999429  0.995873  0.981843   \n",
       "m4  0.994468  0.989472  0.761297  0.999429  1.000000  0.996905  0.984529   \n",
       "m5  0.994312  0.989305  0.760598  0.995873  0.996905  1.000000  0.989534   \n",
       "m6  0.985356  0.973131  0.760832  0.981843  0.984529  0.989534  1.000000   \n",
       "m7  0.984884  0.972206  0.760664  0.981131  0.983897  0.989031  0.999621   \n",
       "m8  0.984758  0.972111  0.760563  0.980880  0.983677  0.988838  0.999607   \n",
       "m9  0.993996  0.991001  0.766020  0.991019  0.991447  0.991499  0.978728   \n",
       "\n",
       "          m7        m8        m9  \n",
       "m0  0.984884  0.984758  0.993996  \n",
       "m1  0.972206  0.972111  0.991001  \n",
       "m2  0.760664  0.760563  0.766020  \n",
       "m3  0.981131  0.980880  0.991019  \n",
       "m4  0.983897  0.983677  0.991447  \n",
       "m5  0.989031  0.988838  0.991499  \n",
       "m6  0.999621  0.999607  0.978728  \n",
       "m7  1.000000  0.999765  0.977965  \n",
       "m8  0.999765  1.000000  0.977677  \n",
       "m9  0.977965  0.977677  1.000000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>m0</th>\n",
       "      <th>m1</th>\n",
       "      <th>m2</th>\n",
       "      <th>m3</th>\n",
       "      <th>m4</th>\n",
       "      <th>m5</th>\n",
       "      <th>m6</th>\n",
       "      <th>m7</th>\n",
       "      <th>m8</th>\n",
       "      <th>m9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.069807</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.156410</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.195708</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.135143</td>\n",
       "      <td>0.110310</td>\n",
       "      <td>0.209002</td>\n",
       "      <td>0.122962</td>\n",
       "      <td>0.115048</td>\n",
       "      <td>0.087671</td>\n",
       "      <td>0.051844</td>\n",
       "      <td>0.054520</td>\n",
       "      <td>0.053801</td>\n",
       "      <td>0.096720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.066254</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>test_5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001283</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>test_6</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005188</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>test_7</td>\n",
       "      <td>0.060201</td>\n",
       "      <td>0.047764</td>\n",
       "      <td>0.078715</td>\n",
       "      <td>0.059908</td>\n",
       "      <td>0.061299</td>\n",
       "      <td>0.044704</td>\n",
       "      <td>0.026788</td>\n",
       "      <td>0.026168</td>\n",
       "      <td>0.026936</td>\n",
       "      <td>0.050600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>test_8</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001787</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>test_9</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005260</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>test_10</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.251970</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>test_11</td>\n",
       "      <td>0.044423</td>\n",
       "      <td>0.032686</td>\n",
       "      <td>0.037145</td>\n",
       "      <td>0.043203</td>\n",
       "      <td>0.041321</td>\n",
       "      <td>0.035115</td>\n",
       "      <td>0.021079</td>\n",
       "      <td>0.019475</td>\n",
       "      <td>0.019616</td>\n",
       "      <td>0.039430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>test_12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.023611</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>test_13</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.033721</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>test_14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004018</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>test_15</td>\n",
       "      <td>0.030801</td>\n",
       "      <td>0.034518</td>\n",
       "      <td>0.012947</td>\n",
       "      <td>0.042643</td>\n",
       "      <td>0.041040</td>\n",
       "      <td>0.035167</td>\n",
       "      <td>0.012031</td>\n",
       "      <td>0.012525</td>\n",
       "      <td>0.013043</td>\n",
       "      <td>0.026571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>test_16</td>\n",
       "      <td>0.455328</td>\n",
       "      <td>0.367165</td>\n",
       "      <td>0.414766</td>\n",
       "      <td>0.360245</td>\n",
       "      <td>0.389054</td>\n",
       "      <td>0.438102</td>\n",
       "      <td>0.521426</td>\n",
       "      <td>0.518923</td>\n",
       "      <td>0.550767</td>\n",
       "      <td>0.374384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>test_17</td>\n",
       "      <td>0.020220</td>\n",
       "      <td>0.018850</td>\n",
       "      <td>0.023649</td>\n",
       "      <td>0.027066</td>\n",
       "      <td>0.023447</td>\n",
       "      <td>0.019514</td>\n",
       "      <td>0.006195</td>\n",
       "      <td>0.005685</td>\n",
       "      <td>0.006239</td>\n",
       "      <td>0.024354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>test_18</td>\n",
       "      <td>0.045291</td>\n",
       "      <td>0.048397</td>\n",
       "      <td>0.038228</td>\n",
       "      <td>0.060599</td>\n",
       "      <td>0.054853</td>\n",
       "      <td>0.046081</td>\n",
       "      <td>0.018941</td>\n",
       "      <td>0.016810</td>\n",
       "      <td>0.016784</td>\n",
       "      <td>0.047732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>test_19</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006196</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ID_code        m0        m1        m2        m3        m4        m5  \\\n",
       "0    test_0  0.000000  0.000000  0.069807  0.000000  0.000000  0.000000   \n",
       "1    test_1  0.000000  0.000000  0.156410  0.000000  0.000000  0.000000   \n",
       "2    test_2  0.000000  0.000000  0.195708  0.000000  0.000000  0.000000   \n",
       "3    test_3  0.135143  0.110310  0.209002  0.122962  0.115048  0.087671   \n",
       "4    test_4  0.000000  0.000000  0.066254  0.000000  0.000000  0.000000   \n",
       "5    test_5  0.000000  0.000000  0.001283  0.000000  0.000000  0.000000   \n",
       "6    test_6  0.000000  0.000000  0.005188  0.000000  0.000000  0.000000   \n",
       "7    test_7  0.060201  0.047764  0.078715  0.059908  0.061299  0.044704   \n",
       "8    test_8  0.000000  0.000000  0.001787  0.000000  0.000000  0.000000   \n",
       "9    test_9  0.000000  0.000000  0.005260  0.000000  0.000000  0.000000   \n",
       "10  test_10  0.000000  0.000000  0.251970  0.000000  0.000000  0.000000   \n",
       "11  test_11  0.044423  0.032686  0.037145  0.043203  0.041321  0.035115   \n",
       "12  test_12  0.000000  0.000000  0.023611  0.000000  0.000000  0.000000   \n",
       "13  test_13  0.000000  0.000000  0.033721  0.000000  0.000000  0.000000   \n",
       "14  test_14  0.000000  0.000000  0.004018  0.000000  0.000000  0.000000   \n",
       "15  test_15  0.030801  0.034518  0.012947  0.042643  0.041040  0.035167   \n",
       "16  test_16  0.455328  0.367165  0.414766  0.360245  0.389054  0.438102   \n",
       "17  test_17  0.020220  0.018850  0.023649  0.027066  0.023447  0.019514   \n",
       "18  test_18  0.045291  0.048397  0.038228  0.060599  0.054853  0.046081   \n",
       "19  test_19  0.000000  0.000000  0.006196  0.000000  0.000000  0.000000   \n",
       "\n",
       "          m6        m7        m8        m9  \n",
       "0   0.000000  0.000000  0.000000  0.000000  \n",
       "1   0.000000  0.000000  0.000000  0.000000  \n",
       "2   0.000000  0.000000  0.000000  0.000000  \n",
       "3   0.051844  0.054520  0.053801  0.096720  \n",
       "4   0.000000  0.000000  0.000000  0.000000  \n",
       "5   0.000000  0.000000  0.000000  0.000000  \n",
       "6   0.000000  0.000000  0.000000  0.000000  \n",
       "7   0.026788  0.026168  0.026936  0.050600  \n",
       "8   0.000000  0.000000  0.000000  0.000000  \n",
       "9   0.000000  0.000000  0.000000  0.000000  \n",
       "10  0.000000  0.000000  0.000000  0.000000  \n",
       "11  0.021079  0.019475  0.019616  0.039430  \n",
       "12  0.000000  0.000000  0.000000  0.000000  \n",
       "13  0.000000  0.000000  0.000000  0.000000  \n",
       "14  0.000000  0.000000  0.000000  0.000000  \n",
       "15  0.012031  0.012525  0.013043  0.026571  \n",
       "16  0.521426  0.518923  0.550767  0.374384  \n",
       "17  0.006195  0.005685  0.006239  0.024354  \n",
       "18  0.018941  0.016810  0.016784  0.047732  \n",
       "19  0.000000  0.000000  0.000000  0.000000  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('./subs/sub_cat_0.9180511510872795.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = 0.\n",
    "for model in models:\n",
    "    preds+=model.predict(test[[c for c in test.columns if c!='ID_code']])/5."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub['target'] = preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_code</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_0</td>\n",
       "      <td>0.001111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_1</td>\n",
       "      <td>0.001691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_2</td>\n",
       "      <td>0.001954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test_3</td>\n",
       "      <td>0.098556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test_4</td>\n",
       "      <td>0.001088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>test_5</td>\n",
       "      <td>0.000653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>test_6</td>\n",
       "      <td>0.000679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>test_7</td>\n",
       "      <td>0.045786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>test_8</td>\n",
       "      <td>0.000656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>test_9</td>\n",
       "      <td>0.000680</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ID_code    target\n",
       "0  test_0  0.001111\n",
       "1  test_1  0.001691\n",
       "2  test_2  0.001954\n",
       "3  test_3  0.098556\n",
       "4  test_4  0.001088\n",
       "5  test_5  0.000653\n",
       "6  test_6  0.000679\n",
       "7  test_7  0.045786\n",
       "8  test_8  0.000656\n",
       "9  test_9  0.000680"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('linear_stacking_{}.csv;'.format(score), index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
